/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package uit.tkorg.coauthorgraph.servlet.crawl;

import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
import java.util.ArrayList;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import uit.tkorg.coauthorgraph.bo.AuthorBO;
import uit.tkorg.coauthorgraph.bo.AuthorPaperBO;
import uit.tkorg.coauthorgraph.bo.ConferenceBO;
import uit.tkorg.coauthorgraph.bo.JournalBO;
import uit.tkorg.coauthorgraph.bo.PaperBO;
import uit.tkorg.coauthorgraph.dto.AuthorDTO;
import uit.tkorg.coauthorgraph.dto.ConferenceDTO;
import uit.tkorg.coauthorgraph.dto.CrawlAuthorDTO;
import uit.tkorg.coauthorgraph.dto.JournalDTO;
import uit.tkorg.coauthorgraph.dto.PaperDTO;
import uit.tkorg.coauthorgraph.property.file.MASearchCrawlPatterns;
import uit.tkorg.coauthorgraph.utility.CommonHTMLProcessor;
import uit.tkorg.coauthorgraph.utility.CommonURLQuery;
import uit.tkorg.coauthorgraph.utility.MASFetcher;
import uit.tkorg.coauthorgraph.utility.MASResultAnalyze;

/**
 *
 * @author Huynh Ngoc Tin
 */
public class MASAutoCrawlingServlet extends HttpServlet {
   
    protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        PrintWriter out = response.getWriter();
        String[] authorList = request.getParameterValues("authorList");
        String depthLevel = request.getParameter("depthLevel");

        ArrayList notCrawlAuthorList = new ArrayList();
        ArrayList alreadyCrawlAuthorList = new ArrayList();
        CrawlAuthorDTO crawlingAuthorDTO = null;
        if (authorList != null) {
            for (int i=0; i<authorList.length; i++) {
                crawlingAuthorDTO = new CrawlAuthorDTO();
                crawlingAuthorDTO.setAuthorName(authorList[i]);
                crawlingAuthorDTO.setCrawlDepthLevel(1);
                notCrawlAuthorList.add(crawlingAuthorDTO);
            }
        }

        // Begin of Crawling & Importing Progress
        while ((notCrawlAuthorList != null) && (notCrawlAuthorList.size() > 0)) {
            crawlingAuthorDTO = (CrawlAuthorDTO) notCrawlAuthorList.get(0);
            // Check if the depth level is reached and stop crawling & importing progress.
            if (!depthLevel.equalsIgnoreCase("Unlimited")) {
                int curCrawlingDepthLevel = crawlingAuthorDTO.getCrawlDepthLevel();
                if (curCrawlingDepthLevel > Integer.parseInt(depthLevel)){
                    notCrawlAuthorList.clear();
                    break;
                }
            }
            
            System.out.println("************************************************************");
            System.out.println("*** Crawling for author:" + crawlingAuthorDTO.getAuthorName());
            System.out.println("*** Current Depth Level:" + crawlingAuthorDTO.getCrawlDepthLevel());
            System.out.println("************************************************************");

            autoCrawlAndImportForAnAuthor(crawlingAuthorDTO.getAuthorName(), depthLevel, request, response);
            alreadyCrawlAuthorList.add(crawlingAuthorDTO);
            
            // get the co-cothor list of the crawling author to add to the list 'notCrawlAuthorList'
            try {
                AuthorPaperBO authorPaperBO = new AuthorPaperBO();
                ArrayList coAuthorList = authorPaperBO.getCoAuthorList(crawlingAuthorDTO.getAuthorName());
                if (coAuthorList != null) {
                    for (int i=0; i<coAuthorList.size(); i++){
                        AuthorDTO curAuthorDTO = (AuthorDTO) coAuthorList.get(i);
                        String curCoAuthorName = curAuthorDTO.getAuthorName();
                        int curCoAuthorID = curAuthorDTO.getAuthorID();
                        // Check if this author exist in the 'notCrawlAuthorList' or not
                        boolean found = false;
                        for (int j=0; j<notCrawlAuthorList.size(); j++) {
                            CrawlAuthorDTO notCrawlAuthorDTO = (CrawlAuthorDTO) notCrawlAuthorList.get(j);
                            if (curCoAuthorName.equalsIgnoreCase(notCrawlAuthorDTO.getAuthorName())) {
                                found = true;
                                break;
                            }
                        }
                        // Check if this author exist in the 'alreadyCrawlAuthorList' or not
                        for (int k=0; k<alreadyCrawlAuthorList.size(); k++) {
                            CrawlAuthorDTO alreadyCrawlAuthorDTO = (CrawlAuthorDTO) alreadyCrawlAuthorList.get(k);
                            if (curCoAuthorName.equalsIgnoreCase(alreadyCrawlAuthorDTO.getAuthorName())) {
                                found = true;
                                break;
                            }
                        }
                        // add a new co-author into the 'notCrawlAuthorList' list.
                        if (found == false) {
                            CrawlAuthorDTO newCrawlAuthorDTO = new CrawlAuthorDTO();
                            newCrawlAuthorDTO.setAuthorName(curCoAuthorName);
                            newCrawlAuthorDTO.setCrawlDepthLevel(crawlingAuthorDTO.getCrawlDepthLevel() + 1);
                            notCrawlAuthorList.add(newCrawlAuthorDTO);
                        }
                    }
                }

                // remove the first CrawlAuthorDTO in the 'notCrawlAuthorList' list
                notCrawlAuthorList.remove(0);
            }
            catch (Exception ex) {
                ex.printStackTrace();
            }
        } // end of while(...) - End of of Crawling & Importing Progress

        // clear all list
        notCrawlAuthorList.clear();
        alreadyCrawlAuthorList.clear();

        System.out.println("************************************************************");
        System.out.println("-------------------DONE DONE DONE --------------------------");
        System.out.println("************************************************************");
        String resultMessage = "Please take a look at the Console to see the crawl & import progress";
        request.setAttribute("resultMessage", resultMessage);
        RequestDispatcher dispatcher = request.getRequestDispatcher("./jsp/MASAutoCrawling.jsp");
        dispatcher.forward(request, response);
    }

    /**
    * Crawl publications & other information relate to an author from MAS
    * @param authorName
    * @param depthLevel
    * @param request
    */
    public void autoCrawlAndImportForAnAuthor(String crawlAuthorName, String depthLevel, HttpServletRequest request, HttpServletResponse response) {
        CommonHTMLProcessor htmlParser = new CommonHTMLProcessor();
        MASFetcher fetcher = new MASFetcher();
        String _pageContent = fetcher.fetchHTMLContentForAuthor(crawlAuthorName);
        int checkStatus = 0;;
        // Collect all publications at returned pages
        try {
            ArrayList publicationList =  MASResultAnalyze.getPublications(_pageContent, crawlAuthorName);
            importMetaDataPublicationForAllCrawledPapers(publicationList);
        }
        catch (Exception ex) {
            ex.printStackTrace();
        }
    }

    /**
     * import a specified list of publications into the DB
     * @param publicationList
     */
    private void importMetaDataPublicationForAllCrawledPapers(ArrayList publicationList) {
        try {
            PaperBO paperBO = new PaperBO();
            AuthorBO authorBO = new AuthorBO();

            PaperDTO paperDTO;
            // Import the metadata information for all papers that crawled from MAS
            System.out.println("*** Number of publications:" + publicationList.size());
            for (int i=0; i<publicationList.size(); i++) {
                paperDTO = (PaperDTO) publicationList.get(i);
                if (!paperBO.isPaperExisted(paperDTO)) { // LOI O DAY
                    System.out.println((i+1) + "." + paperDTO.getTitle() + ".");

                    // insert conference of this paper
                    String confName = paperDTO.getConferenceName();
                    if (confName != null && !confName.equalsIgnoreCase("")) {
                        ConferenceBO confBO = new ConferenceBO();
                        ConferenceDTO confDTO = new ConferenceDTO();
                        confDTO.setConferenceName(confName);
                        confDTO.setYear(paperDTO.getYear());
                        int confID = confBO.getConferenceID(confName, paperDTO.getYear());
                        if (confID == -1) {
                            int insertConfID = confBO.insertNewConference(confDTO);
                            paperDTO.setConferenceID(insertConfID);
                        }
                        else {
                            // this paper relate to a conference that existed inDB
                            paperDTO.setConferenceID(confID);
                        }
                    }

                    // insert journal of this paper
                    String journalName = paperDTO.getJournalName();
                    if (journalName != null && !journalName.equalsIgnoreCase("")) {
                        JournalBO journalBO = new JournalBO();
                        JournalDTO journalDTO = new JournalDTO();
                        journalDTO.setJournalName(journalName);
                        journalDTO.setYear(paperDTO.getYear());
                        int journalID = journalBO.getJournalID(journalName, paperDTO.getYear());
                        if (journalID == -1) {
                            int insertJourID = journalBO.insertNewJournal(journalDTO);
                            paperDTO.setJournalID(insertJourID);
                        }
                        else {
                            // this paper relate to a journal that existed inDB
                            paperDTO.setJournalID(journalID);
                        }
                    }

                    // insert a new paper into the 'paper' table
                    int insertPaperID = paperBO.insertNewPaper(paperDTO);

                    // insert authors into the tables ('author', 'authorinstance', 'authorpaper')
                    ArrayList authors = paperDTO.getAuthorNameList();
                    for (int j=0; j<authors.size(); j++) {
                        String importAuthorName = (String) authors.get(j);
                        importAuthorName = importAuthorName.replaceAll("(\r\n|\r|\n|\n\r)", "");
                        int importAuthorID = authorBO.getAuthorID(importAuthorName);
                        if (importAuthorID == -1) { // not exist in the 'Author' Table
                            // update information for authorDTO object
                            AuthorDTO authorDTO = new AuthorDTO();
                            authorDTO.setAuthorName(importAuthorName);
                            importAuthorID = authorBO.insertNewAuthor(authorDTO);
                        }

                        AuthorPaperBO author_paper_BO = new AuthorPaperBO();
                        if (!author_paper_BO.isAuthorPaperExist(importAuthorID, insertPaperID)) {
                            // insert authorID, paperID into the 'AuthorPaper' table
                            int author_OrderNum = j+1;
                            author_paper_BO.insertNewAuthorPaper(importAuthorID, insertPaperID, author_OrderNum);
                        }
                    }
                }
                else { // the paper is existed
                    System.out.println((i+1) + "." + paperDTO.getTitle() + ". " + "EXISTED, NOT INSERT!");
                }
            } // end of for (...)
        }
        catch (Exception ex) {
            ex.printStackTrace();
        }
    }
}
